# 抓取起点中文网的小说信息
#zgh
# 破解起点中文版  "字数"信息 使用自定义字体反爬虫手段 字体 使用 TTFont


import time

import requests
from lxml import etree
from lxml.etree import _Element
import xlwt
from xlwt.Worksheet import Worksheet
import re
import os
import random
from fontTools.ttLib import TTFont

urls = ["https://www.qidian.com/all/page{}/".format(str(page)) for page in range(1, 6)]

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}

session = requests.session()

data = []


def getHtmlContent(url):
    response = session.get(url, headers=headers)
    if (response.status_code == requests.codes.ok):
        html = response.content.decode("utf-8")
        # with open("qidian.html", "w", encoding="utf-8") as fp:
        #     fp.write(html)
        return html

def parseHtml(html):
    root = etree.HTML(html)
    root: _Element
    for li_row in root.xpath("//ul[contains(@class,'all-img-list')]/li"):
        li_row: _Element
        li_str = str(etree.tostring(li_row, pretty_print=True), encoding="utf-8")
        style = ''
        if (len(li_row.xpath(".//p[@class='author']/a[2]")) > 0):
            style += li_row.xpath(".//p[@class='author']/a[2]")[0].text
        if (len(li_row.xpath(".//p[@class='author']/a[3]")) > 0):
            style += '·' + li_row.xpath(".//p[@class='author']/a[3]")[0].text
        wof = re.findall(r"src: url\(\'(.*?)\'\)", li_str, re.S)[1]
        font_name = wof.split("/")[-1]
        # 保存字体文件
        if not os.path.exists("font/" + font_name):
            wof_response = session.get(wof, headers=headers)
            with open("font/" + font_name, 'wb') as wof_handler:
                wof_handler.write(wof_response.content)
        font = TTFont("font/" + font_name)
        font.saveXML("font/" + (font_name.split(".")[0]) + ".xml")
        font_map = font['cmap'].getBestCmap()
        # 修改字体映射
        # 找到字体映射关系，并手动修改后的字体映射
        change_map = {'three': 3, 'eight': 8, 'four': 4, 'two': 2, 'zero': 0, 'seven': 7, 'six': 6, 'five': 5, 'one': 1,
                      'period': '.',
                      'nine': 9}
        for mapkey, mapval in font_map.items():
            font_map[mapkey] = change_map[mapval]

        for key in font_map:
            li_str = li_str.replace('&#' + str(key) + ';', str(font_map[key]))

        # 获取作品的字数
        # </style><span class="BMvPuniI">256.43</span>&#19975;&#23383;</span>
        word_group = re.search(r'</style><span class=".*?">(.*?)</span>', li_str, re.S)
        if word_group is not None:
            word = word_group.group(1)
        else:
            word = ''
        data.append({
            "name": li_row.xpath(".//h4/a/text()")[0],
            "style": style,
            "complete": li_row.xpath(".//p[@class='author']/span/text()")[0],
            "author": li_row.xpath(".//p[@class='author']/a[@class='name']/text()")[0],
            "intro": li_row.xpath(".//p[@class='intro']/text()")[0].strip(),
            "words": word + "万字"
        })


def saveData(data):
    excel = xlwt.Workbook(encoding="utf-8")
    sheet: Worksheet = excel.add_sheet("起点中文网小说信息")
    excel_header = ["小说名称", "作者", "类型", "完成度", "字数", "内容"]
    for index, h in enumerate(excel_header):
        sheet.write(0, index, h)
    for key, row in enumerate(data):
        sheet.write(key + 1, 0, row['name'])
        sheet.write(key + 1, 1, row['author'])
        sheet.write(key + 1, 2, row['style'])
        sheet.write(key + 1, 3, row['complete'])
        sheet.write(key + 1, 4, row['words'])
        sheet.write(key + 1, 5, row['intro'])
    excel.save("起点中文网小说信息.xls")


# 执行函数

def do_exec():
    for url in urls:
        html = getHtmlContent(url)
        parseHtml(html)
        # 随机停顿 秒数 防止
        time.sleep(random.randint(0, 6))
    saveData(data)


do_exec()
